"""
Case Type   : ustore_DQL_SET_OPERATIONS
Case Name   : 多个union结合使用
Create At   : 2022-05-12
Owner       : nanyang12
Description :
    1.创建表
    2.使用copy命令插入数据
    3.多个union结合使用
    4.环境清理
Expect      :
    1.创建成功
    2.数据插入成功
    3.成功
    4.清理成功
History     :
"""

import os
import subprocess
import unittest

from testcase.utils.Common import Common
from testcase.utils.CommonSH import CommonSH
from testcase.utils.Constant import Constant
from testcase.utils.Logger import Logger
from yat.test import Node
from yat.test import macro


class Ustore01(unittest.TestCase):
    def setUp(self):
        self.log = Logger()
        self.log.info(f'----- {os.path.basename(__file__)} start-----')
        self.pri_sh = CommonSH('PrimaryDbUser')
        self.pri_root = Node('PrimaryRoot')
        self.com = Common()
        self.constant = Constant()
        self.table_name1 = 't_ustore_Cast_mult_0001_01'
        self.table_name2 = 't_ustore_Cast_mult_0001_02'
        self.table_name3 = 't_ustore_Cast_mult_0001_03'
        self.table_name4 = 't_ustore_Cast_mult_0001_04'
        self.table_name5 = 't_ustore_Cast_mult_0001_05'
        self.copy_file1 = 'date_dim_33.txt'
        self.copy_file2 = 'store_33.txt'
        self.copy_file3 = 'customer_33.txt'
        self.copy_file4 = 'call_center_33.txt'
        self.copy_file5 = 'web_site_33.txt'
        self.cur_path_1 = os.path.join(macro.SCRIPTS_PATH, self.copy_file1)
        self.cur_path_2 = os.path.join(macro.SCRIPTS_PATH, self.copy_file2)
        self.cur_path_3 = os.path.join(macro.SCRIPTS_PATH, self.copy_file3)
        self.cur_path_4 = os.path.join(macro.SCRIPTS_PATH, self.copy_file4)
        self.cur_path_5 = os.path.join(macro.SCRIPTS_PATH, self.copy_file5)
        self.parent_path = os.path.dirname(macro.DB_INSTANCE_PATH)
        self.target_path1 = os.path.join(self.parent_path, self.copy_file1)
        self.target_path2 = os.path.join(self.parent_path, self.copy_file2)
        self.target_path3 = os.path.join(self.parent_path, self.copy_file3)
        self.target_path4 = os.path.join(self.parent_path, self.copy_file4)
        self.target_path5 = os.path.join(self.parent_path, self.copy_file5)

    def test_join(self):
        step_txt = '-----step1:创建表;expect: 创建成功-----'
        self.log.info(step_txt)
        cmd_sql = f"""drop table if exists {self.table_name1}  CASCADE ;
                    drop table if exists {self.table_name2}  CASCADE ;
                    drop table if exists {self.table_name3}  CASCADE ;
                    drop table if exists {self.table_name4}  CASCADE ;
                    drop table if exists {self.table_name5}  CASCADE ;
        create table {self.table_name1}
        (
            d_date_sk                 integer               not null,
            d_date_id                 char(16)              not null,
            d_date                    date                          ,
            d_month_seq               integer                       ,
            d_week_seq                integer                       ,
            d_quarter_seq             integer                       ,
            d_year                    integer                       ,
            d_dow                     integer                       ,
            d_moy                     integer                       ,
            d_dom                     integer                       ,
            d_qoy                     integer                       ,
            d_fy_year                 integer                       ,
            d_fy_quarter_seq          integer                       ,
            d_fy_week_seq             integer                       ,
            d_day_name                char(9)                       ,
            d_quarter_name            char(6)                       ,
            d_holiday                 char(1)                       ,
            d_weekend                 char(1)                       ,
            d_following_holiday       char(1)                       ,
            d_first_dom               integer                       ,
            d_last_dom                integer                       ,
            d_same_day_ly             integer                       ,
            d_same_day_lq             integer                       ,
            d_current_day             char(1)                       ,
            d_current_week            char(1)                       ,
            d_current_month           char(1)                       ,
            d_current_quarter         char(1)                       ,
            d_current_year            char(1)                       
         )
        WITH (STORAGE_TYPE=USTORE) partition by Range(d_year) (
         partition p1 values less than(1950),
         partition p2 values less than(2000),
         partition p3 values less than(2050),
         partition p4 values less than(2100),
         partition p5 values less than(3000),
         partition p6 values less than(maxvalue)
        );
        create table {self.table_name2}
        (
            s_store_sk                integer               not null,
            s_store_id                char(16)              not null,
            s_rec_start_date          date                          ,
            s_rec_end_date            date                          ,
            s_closed_date_sk          integer                       ,
            s_store_name              varchar(50)                   ,
            s_number_employees        integer                       ,
            s_floor_space             integer                       ,
            s_hours                   char(20)                      ,
            s_manager                 varchar(40)                   ,
            s_market_id               integer                       ,
            s_geography_class         varchar(100)                  ,
            s_market_desc             varchar(100)                  ,
            s_market_manager          varchar(40)                   ,
            s_division_id             integer                       ,
            s_division_name           varchar(50)                   ,
            s_company_id              integer                       ,
            s_company_name            varchar(50)                   ,
            s_street_number           varchar(10)                   ,
            s_street_name             varchar(60)                   ,
            s_street_type             char(15)                      ,
            s_suite_number            char(10)                      ,
            s_city                    varchar(60)                   ,
            s_county                  varchar(30)                   ,
            s_state                   char(2)                       ,
            s_zip                     char(10)                      ,
            s_country                 varchar(20)                   ,
            s_gmt_offset              decimal(5,2)                  ,
            s_tax_precentage          decimal(5,2)                  
         )  WITH (STORAGE_TYPE=USTORE);
        create table {self.table_name3}
        (
            c_customer_sk             integer               not null,
            c_customer_id             char(16)              not null,
            c_current_cdemo_sk        integer                       ,
            c_current_hdemo_sk        integer                       ,
            c_current_addr_sk         integer                       ,
            c_first_shipto_date_sk    integer                       ,
            c_first_sales_date_sk     integer                       ,
            c_salutation              char(10)                      ,
            c_first_name              char(20)                      ,
            c_last_name               char(30)                      ,
            c_preferred_cust_flag     char(1)                       ,
            c_birth_day               integer                       ,
            c_birth_month             integer                       ,
            c_birth_year              integer                       ,
            c_birth_country           varchar(20)                   ,
            c_login                   char(13)                      ,
            c_email_address           char(50)                      ,
            c_last_review_date        char(10)
        )  WITH (STORAGE_TYPE=USTORE);
        create table {self.table_name4}
        (
          cc_call_center_sk         integer               not null,
          cc_call_center_id         char(16)              not null,
          cc_rec_start_date         date                          ,
          cc_rec_end_date           date                          ,
          cc_closed_date_sk         integer                       ,
          cc_open_date_sk           integer                       ,
          cc_name                   varchar(50)                   ,
          cc_class                  varchar(50)                   ,
          cc_employees              integer                       ,
          cc_sq_ft                  integer                       ,
          cc_hours                  char(20)                      ,
          cc_manager                varchar(40)                   ,
          cc_mkt_id                 integer                       ,
          cc_mkt_class              char(50)                      ,
          cc_mkt_desc               varchar(100)                  ,
          cc_market_manager         varchar(100)                   ,
          cc_division               varchar(100)                       ,
          cc_division_name          varchar(50)                   ,
          cc_company                varchar(100)                       ,
          cc_company_name           char(50)                      ,
          cc_street_number          char(10)                      ,
          cc_street_name            varchar(60)                   ,
          cc_street_type            char(15)                      ,
          cc_suite_number           char(10)                      ,
          cc_city                   varchar(60)                   ,
          cc_county                 varchar(30)                   ,
          cc_state                  char(50)                       ,
          cc_zip                    char(20)                      ,
          cc_country                varchar(20)                   ,
          cc_gmt_offset             decimal(5,2)                  ,
          cc_tax_percentage         decimal(5,2)                  
        )  WITH (STORAGE_TYPE=USTORE);
        create table {self.table_name5}
        (
            web_site_sk               integer               not null,
            web_site_id               char(16)              not null,
            web_rec_start_date        date                          ,
            web_rec_end_date          date                          ,
            web_name                  varchar(50)                   ,
            web_open_date_sk          integer                       ,
            web_close_date_sk         integer                       ,
            web_class                 varchar(50)                   ,
            web_manager               varchar(40)                   ,
            web_mkt_id                integer                       ,
            web_mkt_class             varchar(50)                   ,
            web_mkt_desc              varchar(100)                  ,
            web_market_manager        varchar(100)                   ,
            web_company_id            integer                  ,
            web_company_name          char(50)                      ,
            web_street_number         char(10)                      ,
            web_street_name           varchar(60)                   ,
            web_street_type           char(15)                      ,
            web_suite_number          char(50)                      ,
            web_city                  varchar(60)                   ,
            web_county                varchar(30)                   ,
            web_state                 char(20)                       ,
            web_zip                   char(50)                      ,
            web_country               varchar(20)                   ,
            web_gmt_offset            decimal(5,2)                  ,
            web_tax_percentage        decimal(5,2)                  
         )
         WITH (STORAGE_TYPE=USTORE) partition by range(web_rec_start_date)
        (
         partition p1 values less than('1990-01-01'),
         partition p2 values less than('1995-01-01'),
         partition p3 values less than('2000-01-01'),
         partition p4 values less than('2005-01-01'),
         partition p5 values less than('2010-01-01'),
         partition p6 values less than('2015-01-01'),
         partition p7 values less than(maxvalue)
        );"""
        self.log.info(cmd_sql)
        create_result = self.pri_sh.execut_db_sql(cmd_sql)
        self.log.info(create_result)
        self.assertTrue(
            create_result.count(self.constant.TABLE_CREATE_SUCCESS) == 5,
            '执行失败:' + step_txt)

        step_txt = '-----step2:使用copy命令插入数据，expect: 数据插入成功-----'
        self.log.info(step_txt)
        text = '-----step2.1: 获取当前主机的IP地址;expect: 成功-----'
        self.log.info(text)
        cmd = "ifconfig -a|grep inet6 -a2|grep broadcast|awk '{print $2}'"
        self.log.info(cmd)
        self.cur_ip = subprocess.getoutput(cmd).strip()
        self.log.info(self.cur_ip)

        text = '-----step2.2: 复制文件到数据库主机对应目录 expect: 成功-----'
        self.log.info(text)
        if self.cur_ip == self.pri_sh.node.ssh_host:
            self.log.info('当前主机是数据库主机，无需远程复制')
            cmd = f"cp {self.cur_path_1} {self.parent_path};" \
                f"cp {self.cur_path_2} {self.parent_path};" \
                f"cp {self.cur_path_3} {self.parent_path};" \
                f"cp {self.cur_path_4} {self.parent_path};" \
                f"cp {self.cur_path_5} {self.parent_path};"
            self.log.info(cmd)
            res = self.pri_sh.node.sh(cmd).result()
            self.log.info(res)
        else:
            self.com.scp_file(self.pri_root, self.copy_file1,
                              self.parent_path)
            self.com.scp_file(self.pri_root, self.copy_file2,
                              self.parent_path)
            self.com.scp_file(self.pri_root, self.copy_file3,
                              self.parent_path)
            self.com.scp_file(self.pri_root, self.copy_file4,
                              self.parent_path)
            self.com.scp_file(self.pri_root, self.copy_file5,
                              self.parent_path)
            chmod_cmd = f"chmod 777 -R {self.parent_path};" \
                        f"ls -al {self.parent_path}"
            self.log.info(chmod_cmd)
            chmod_result = self.pri_root.sh(chmod_cmd).result()
            self.log.info(chmod_result)

        text = '-----step2.3: 查看数据库主机目标路径下文件 expect: 成功-----'
        self.log.info(text)
        cmd = f"ls -l {self.parent_path}| grep -E '{self.copy_file1}';" \
            f"ls -l {self.parent_path}| grep -E '{self.copy_file2}';" \
            f"ls -l {self.parent_path}| grep -E '{self.copy_file3}';" \
            f"ls -l {self.parent_path}| grep -E '{self.copy_file4}';" \
            f"ls -l {self.parent_path}| grep -E '{self.copy_file5}';"
        self.log.info(cmd)
        res = self.pri_root.sh(cmd).result()
        self.log.info(res)
        self.assertNotIn('No such file or directory', res, '文件拷贝失败')
        self.assertIn(f'{self.copy_file1}', res, '文件拷贝失败')
        self.assertIn(f'{self.copy_file2}', res, '文件拷贝失败')
        self.assertIn(f'{self.copy_file3}', res, '文件拷贝失败')
        self.assertIn(f'{self.copy_file4}', res, '文件拷贝失败')
        self.assertIn(f'{self.copy_file5}', res, '文件拷贝失败')

        text = '-----step2.4:使用copy命令插入数据，expect: 数据插入成功-----'
        self.log.info(text)
        self.log.info('---------------使用copy命令插入数据------------------')
        copy_sql = f"copy {self.table_name1} from '{self.target_path1}' " \
            f"delimiters ',';" \
            f"copy {self.table_name2} from '{self.target_path2}' " \
            f"delimiters ',';" \
            f"copy {self.table_name3} from '{self.target_path3}' " \
            f"delimiters ',';" \
            f"copy {self.table_name3} from '{self.target_path4}' " \
            f"delimiters ',';" \
            f"copy {self.table_name3} from '{self.target_path5}' " \
            f"delimiters ',';"
        self.log.info(copy_sql)
        copy_result = self.pri_sh.execut_db_sql(copy_sql)
        self.log.info(copy_result)
        self.assertIn('COPY 12', copy_result, '执行失败:' + step_txt)
        self.assertIn('COPY 10000', copy_result, '执行失败:' + step_txt)
        self.assertIn('COPY 73049', copy_result, '执行失败:' + step_txt)

        step_txt = '-----step3: 多个union结合使用 expect: 成功-----'
        self.log.info(step_txt)
        cmd_sql = f"""
            select max(d_quarter_seq),
                   sum(d_fy_quarter_seq),
                   avg(d_dow),
                   count(*),
                   d_date
              from {self.table_name1}
             where d_fy_year = 2000
             group by d_date
            union all
            select s_market_id,
                   s_company_id,
                   s_gmt_offset,
                   max(s_store_sk),
                   s_rec_start_date
              from {self.table_name2}
             where s_rec_end_date is not null
             group by s_gmt_offset, cube(s_market_id,
                   s_company_id), s_rec_start_date
            union (select c_birth_day,
                          coalesce(length(trim(c_first_name)),
                                   length(trim(c_last_name)),
                                   max(c_first_shipto_date_sk)),
                          '' ::numeric,
                          -1 ::smallint,
                          max(cast((c_birth_year  || '-' || c_birth_month
                                     || '-' || c_birth_day ) as date))
                     from {self.table_name3}
                    where c_birth_day = c_birth_month
                      and c_birth_day is not null
                      and c_birth_year is not null
                      and c_birth_month is not null
                    group by c_birth_day,
                             length(trim(c_first_name)),
                             length(trim(c_last_name))
                    order by 1, 2, 3, 4, 5 limit 10)
            union
            select 1, null, cc_sq_ft / cc_employees, length(cc_mkt_desc), ''
              from {self.table_name4}
             where cc_rec_end_date is null 
            except
              select case when web_mkt_id <> 1 
                          then 403 ::smallint
                          else web_mkt_id
                     end,
                     case when 400 + web_site_sk - 10 < 400 
                          then 403
                          when 400 + web_site_sk - 10 > 403 
                          then 403
                          else 400 + web_site_sk - 10
                     end,
                     case when web_mkt_id > 3 
                          then web_mkt_id - 2
                          else web_mkt_id
                     end,
                     grouping(web_open_date_sk) + 1,
                     case when web_rec_end_date is null 
                          then '2000-8-15' ::date
                          when web_rec_end_date = '2000-8-15' ::date 
                          then web_rec_end_date + 7
                          else '2000-8-8' ::date
                     end
            from {self.table_name5}
            where -substr(web_open_date_sk, -1) < -1 ::smallint
            group by rollup(case when web_mkt_id <> 1 
                          then 403 ::smallint
                          else web_mkt_id
                     end, case when web_mkt_id > 3 
                          then web_mkt_id - 2
                          else web_mkt_id
                     end), cube(case when 400 + web_site_sk - 10 < 400 
                          then 403
                          when 400 + web_site_sk - 10 > 403 
                          then 403
                          else 400 + web_site_sk - 10
                     end, case when web_rec_end_date is null 
                          then '2000-8-15' ::date
                          when web_rec_end_date = '2000-8-15' ::date 
                          then web_rec_end_date + 7
                          else '2000-8-8' ::date
                     end), web_open_date_sk
            order by 1,2,3,4,5 limit 20;"""
        self.log.info(cmd_sql)
        query_result = self.pri_sh.execut_db_sql(cmd_sql)
        self.log.info(query_result)
        self.assertIn('20 rows', query_result, '执行失败:' + step_txt)

    def tearDown(self):
        step_txt = '-----step4: 环境清理 expect:清理成功-----'
        self.log.info(step_txt)
        self.log.info('删除表')
        cmd_sql = self.pri_sh.execut_db_sql(f"""
                    drop table {self.table_name1} cascade;
                    drop table {self.table_name2} cascade;
                    drop table {self.table_name3} cascade;
                    drop table {self.table_name4} cascade;
                    drop table {self.table_name5} cascade;""")
        self.log.info(cmd_sql)
        text = '-----删除文件-----'
        self.log.info(text)
        rm_cmd = f"rm -rf {self.target_path1};" \
            f"rm -rf {self.target_path2};" \
            f"rm -rf {self.target_path3};" \
            f"rm -rf {self.target_path4};" \
            f"rm -rf {self.target_path5};"
        self.log.info(rm_cmd)
        self.com.get_sh_result(self.pri_root, rm_cmd)
        for s in [self.target_path1, self.target_path2]:
            cmd = f'if [ -f {s} ]; then echo "does exists"; ' \
                f'else echo "not exists"; fi'
            file_judgments = self.com.get_sh_result(self.pri_root, cmd)
            self.assertIn('not exists', file_judgments, "清理失败" + text)
        self.assertTrue(
            cmd_sql.count(self.constant.DROP_TABLE_SUCCESS) == 5,
            '执行失败:' + step_txt)
        self.log.info(f'----- {os.path.basename(__file__)} end-----')
